﻿using System;
using System.Data;
using System.Configuration;
using System.Net;
using System.Text.RegularExpressions;
using System.Text;
using System.Web.UI;
using System.Web.UI.WebControls;
using System.Web;
using System.IO;
using System.Collections;



/// <summary>
///getData 的摘要说明
/// </summary>
public class GetData
{
    /// <summary>
    /// 从指定网址中获取网页内容。
    /// </summary>
    /// <param name="url">网址</param>
    /// <returns></returns>
    public static string getHtml(string url)
    {
        WebClient myWebClient = new WebClient();

        myWebClient.Credentials = CredentialCache.DefaultCredentials;
        //获得数据
        byte[] myDataBuffer = myWebClient.DownloadData(url);
        string strWebData = Encoding.Default.GetString(myDataBuffer);

        //获取网页字符编码描述信息 
        Match charSetMatch = Regex.Match(strWebData, "<meta([^<]*)charset=([^<]*)\"", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        string charSet = charSetMatch.Groups[2].Value;

        if (charSet != null && charSet != "" && Encoding.GetEncoding(charSet) != Encoding.Default)
            strWebData = Encoding.GetEncoding(charSet).GetString(myDataBuffer);
        return strWebData;
    }
    /// <summary>
    /// 从指定网址中获取网页内容。
    /// </summary>
    /// <param name="url">网址</param>
    /// <param name="charSet">指定的字符编码格式</param>
    /// <returns>网页源代码</returns>
    public static string getHtml(string url, string charSet)
    //url是要访问的地址，charSet是目标网页的编码，如果传入的是null或者""，那就自动分析网页的编码 
    {

        WebClient myWebClient = new WebClient();
        myWebClient.Credentials = CredentialCache.DefaultCredentials;
        //获得数据
        byte[] myDataBuffer = myWebClient.DownloadData(url);
        string strWebData = Encoding.Default.GetString(myDataBuffer);

        if (charSet != null && charSet != "" && Encoding.GetEncoding(charSet) != Encoding.Default)
            strWebData = Encoding.GetEncoding(charSet).GetString(myDataBuffer);
        return strWebData;
    }
    /// <summary>
    /// 包含参数的获取,如果有中文字符，必须先编码为与目标网页一致
    /// </summary>
    /// <param name="url">地址</param>
    /// <param name="param">参数</param>
    /// <param name="charSet">字符串格式</param>
    /// <param name="method">Post或者Get</param>
    /// <returns>网页内容</returns>
    public static string getHtml(string url,string param,string charSet,string method)
    {
        //默认为POST方法,utf-8编码
        if (method == "")
            method = "POST";
        if (charSet == "")
            charSet = "uft-8";

        WebClient myWebClient = new WebClient();
        //构建参数的比特串
        byte[] postData = Encoding.ASCII.GetBytes(param);
        //设置头
        myWebClient.Headers.Add("Content-Type", "application/x-www-form-urlencoded");
        myWebClient.Headers.Add("ContentLength", postData.Length.ToString());
        //提交
        byte[] myDataBuffer = myWebClient.UploadData(url, method, postData);
        //解码
        return Encoding.GetEncoding(charSet).GetString(myDataBuffer);
       
    }
    /// <summary>
    /// 根据服务器名称获得所有的正则表达式
    /// </summary>
    /// <param name="sHotName">服务器名称</param>
    /// <returns>正则表达式列表</returns>
    public static RegList getRegList(string sHostName)
    {
        RegList reglist = new RegList();
        SqlDataSource ds = new SqlDataSource();
        ds.ConnectionString = System.Web.Configuration.WebConfigurationManager.ConnectionStrings["ls_Database"].ToString();
        ds.SelectCommand = "select * from ls_Site where Site_Index=@siteindex";
        ds.SelectParameters.Add("siteindex", sHostName);
        DataView dv = (DataView)ds.Select(DataSourceSelectArguments.Empty);
        if (dv.Count > 0)
        {
            reglist.SiteName = dv[0]["Site_Name"].ToString();
            reglist.RegBookName = dv[0]["Site_NameReg"].ToString();
            reglist.RegChapterList = dv[0]["Site_ListReg"].ToString();
            reglist.RegTitle = dv[0]["Site_TitleReg"].ToString();
            reglist.RegContent = dv[0]["Site_ContentReg"].ToString();
            reglist.RegExcept = dv[0]["Site_ExceptReg"].ToString();
            reglist.DefaultIndex = dv[0]["Site_DefaultIndex"].ToString();
        }
        ds.Dispose();
        dv.Dispose();
        return reglist;
    }
    /// <summary>
    /// 获得标题部分
    /// </summary>
    /// <param name="sHtml">源文件</param>
    /// <param name="sRegTitle">处理得正则表达式</param>
    /// <returns>标题字符串</returns>
    public static string getTitle(string sHtml,string sRegTitle)
    {
        string sTitle = string.Empty;
        Match match = Regex.Match(sHtml, sRegTitle, RegexOptions.IgnoreCase | RegexOptions.Multiline);
        if (match.Success)
        {
            sTitle = match.Groups[1].Value;
        }
        return sTitle;
    }
   /// <summary>
   /// 获得内容
   /// </summary>
   /// <param name="sHostName">主机名</param>
   /// <param name="sHtml">HTML源代码</param>
   /// <param name="sRegConent">处理的正则表达</param>
   /// <param name="sRegExcept">需要特别处理的字符串</param>
   /// <returns>处理后结果</returns>
    public static string getContent(string sHostName,string sHtml, string sRegContent,string sRegExcept)
    {
        string sContent = string.Empty;
        string sOldImageURL = string.Empty;
        Match match = Regex.Match(sHtml, sRegContent, RegexOptions.IgnoreCase | RegexOptions.Multiline);
        if (match.Success)
            sContent = match.Groups[1].Value;
        else//如果没有成功，就以BODY再来一次
        {
            match = Regex.Match(sHtml, "<[^<>]*BODY[^<>]*>([^\\v]*)<[^<>]*/BODY>[^<>]*", RegexOptions.IgnoreCase | RegexOptions.Multiline);
            sContent = match.Groups[1].Value;//在不匹配就没办法了。
        }
        //图片的绝对地址变为全地址。
        // <img src="/DownFiles/Book/11/11423/2009/3/9/20090309111857282.gif" align="center" border=0>
        match = Regex.Match(sHtml, "<IMG[^<>s]*src=\"(/[^\"'>]*)\"[^>]*>", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        //清理一下截取出来的内容
        while (match.Success)
        {
            sOldImageURL = match.Groups[1].Value;
            sContent = sContent.Replace(sOldImageURL, sHostName + sOldImageURL);

            match = match.NextMatch();
        }

        sContent = clearHTML(sContent, sRegExcept);
        return sContent;
    }
    /// <summary>
    /// 清理HTML源代码，去掉一些乱七八糟的格式
    /// </summary>
    /// <param name="sHtml">HTML源代码字符串</param>
    /// <param name="sRegContentExcept">自定义的删除内容</param>
    /// <returns>清理后的字符串</returns>
    public static string clearHTML(string sHtml, string sRegContentExcept)
    {

        //需要去掉的内容
        if (sRegContentExcept != "")
            sHtml = Regex.Replace(sHtml, sRegContentExcept, "", RegexOptions.IgnoreCase | RegexOptions.Multiline);

        sHtml = Regex.Replace(sHtml, "<[ /]*CENTER[^<>]*>", "", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        sHtml = Regex.Replace(sHtml, "<[ /]*UL[^<>]*>", "", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        sHtml = Regex.Replace(sHtml, "<[ /]*LI[^<>]*>", "", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        sHtml = Regex.Replace(sHtml, "<[ /]*SPAN[^<>]*>", "", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        sHtml = Regex.Replace(sHtml, "<[ ]*HR[^<>]*>", "", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        sHtml = Regex.Replace(sHtml, "<!--[^\\f]*?-->", "", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        sHtml = Regex.Replace(sHtml, "<[ /]*TABLE[^<>]*>", "", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        sHtml = Regex.Replace(sHtml, "<[ /]*FONT[^<>]*>", "", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        sHtml = Regex.Replace(sHtml, "<[ /]*TR[^<>]*>", "", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        sHtml = Regex.Replace(sHtml, "<[ /]*TD[^<>]*>", "", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        sHtml = Regex.Replace(sHtml, "<[ /]*BR[^<>]*>", "<P>", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        sHtml = Regex.Replace(sHtml, "<[ /]*STRONG>", "<P>", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        sHtml = Regex.Replace(sHtml, "&[a-z0-9#]{3,4};", "", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        sHtml = Regex.Replace(sHtml, "<[^<>]*DIV[^<>]*>", "<P>", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        sHtml = Regex.Replace(sHtml, "<[^<>]*SCRIPT[^<>]*>[^<>]*<[^<>]*/SCRIPT[^<>]*>", "", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        sHtml = Regex.Replace(sHtml, "<[^<>]*FIELDSET[^<>]*>[^\v]*<[^<>]*/FIELDSET[^<>]*>", "", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        sHtml = Regex.Replace(sHtml, "<[^<>]*IFRAM[^<>]*>[^<>]*<[^<>]*/IFRAM[^<>]*>", "", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        sHtml = Regex.Replace(sHtml, "<[^<>]*SCRIPT[^<>]*>", "", RegexOptions.IgnoreCase | RegexOptions.Multiline);//去掉还没有配对的script
        sHtml = Regex.Replace(sHtml, "<[^<>]*A[^<>]*>[^<>]*<[^<>]*/A[^<>]*>", "", RegexOptions.IgnoreCase | RegexOptions.Multiline);//所有的<A></A>
        sHtml = Regex.Replace(sHtml, "<[ /]*A[^<>]*>", "<P>", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        sHtml = Regex.Replace(sHtml, "<[ /]*P[^<>]*>", "<P>", RegexOptions.IgnoreCase | RegexOptions.Multiline);//不同的段落格式装换为一种
        sHtml = Regex.Replace(sHtml, "[\\f\\n\\t\\r\\v]*", "", RegexOptions.IgnoreCase | RegexOptions.Multiline);//去掉所有空白符号
        sHtml = Regex.Replace(sHtml, "<P>[<P> ]*<P>", "<P>", RegexOptions.IgnoreCase | RegexOptions.Multiline);//多个连续的仅放一个
        return sHtml;
    }
    /// <summary>
    /// 建立内容页
    /// </summary>
    /// <param name="sURL">网址</param>
    /// <param name="reglist">正则表达式</param>
    public static void  buildPage(string sURL, RegList reglist,IndexInfo info)
    {
        //书籍根目录
        string sBookRoot = ConfigurationManager.AppSettings["booktemp"];
        //页面模板地址
        string sChapterModel = "/model/ModelChapter.htm";
        //页面模板物理地址
        string sChapterModelMapPath = HttpContext.Current.Server.MapPath(sChapterModel);
        //页面名称
        string sPageName = URLSub.getPageName(sURL);
        //主机名称
        string sHostName = URLSub.getHostName(sURL);
        //网页目录
        string sPathName = URLSub.getPathName(sURL);

        //目录页地址
        string sIndexURL="http://"+sPathName+reglist.DefaultIndex;
        //获得映射地址即C:\website\格式。
        string sFileMapPath = HttpContext.Current.Server.MapPath(sBookRoot + sPathName + sPageName);
        //页面存储路径
        string sDirMapPath = HttpContext.Current.Server.MapPath(sBookRoot + sPathName);
        string sHtml = GetData.getHtml(sURL);
        ////分别获得网页的标题部分和内容部分。
        string sTitle = GetData.getTitle(sHtml, reglist.RegTitle);
        string sContent = GetData.getContent(sHostName, sHtml, reglist.RegContent, reglist.RegExcept);

        StreamReader sr = new StreamReader(sChapterModelMapPath);
        string sModel = sr.ReadToEnd();
        sr.Close();
        //替换

        GuideURL guide= GetData.getGuideURL(sPageName,info);

        string sPageProcess = "/PageClear.aspx?pagename=" + "http://" + sPathName;

        sModel = sModel.Replace("<#PAGE>", sPageProcess);
        sModel = sModel.Replace("<#BOOK>", info.BookName);
        sModel = sModel.Replace("<#TITLE>", sTitle);
        sModel = sModel.Replace("<#CONTENT>", sContent);
        sModel = sModel.Replace("<#LAST>",guide.LastPage );
        sModel = sModel.Replace("<#NEXT>", guide.NextPage);
        sModel = sModel.Replace("<#INDEX>", guide.IndexPage);
        sModel = sModel.Replace("<#END>", guide.EndPage);
        //检查是否存在目录
        if (!Directory.Exists(sDirMapPath))
            Directory.CreateDirectory(sDirMapPath);
        //保存
        StreamWriter sw = new StreamWriter(sFileMapPath, false, Encoding.UTF8);
        sw.WriteLine(sModel);
        sw.Close();
        
    }
    /// <summary>
    /// 建立主页
    /// </summary>
    /// <param name="sIndexURL">主页地址</param>
    /// <param name="reglist">正则表达式列表</param>
    /// <returns>书籍名称</returns>
    public static string buildIndex(string sIndexURL,RegList reglist,IndexInfo info)
    {
        string sBookRoot = ConfigurationManager.AppSettings["booktemp"];
        string sListModel = "/model/ModelIndex.htm";

        //网页名称
        string sPageName = URLSub.getPageName(sIndexURL);
        //主机名称
        string sHostName = URLSub.getHostName(sIndexURL);
        //网页目录
        string sPathName = URLSub.getPathName(sIndexURL);
        
        string sFileMapPath = HttpContext.Current.Server.MapPath(sBookRoot + sPathName + sPageName);
        string sDirMapPath = HttpContext.Current.Server.MapPath(sBookRoot + sPathName);
        string sListModelMapPath = HttpContext.Current.Server.MapPath(sListModel);

        //获得目录页内容
        string sHtml=GetData. getHtml(sIndexURL);
        //获得文章标题。这个将作为书名返回。
        string sBookName = GetData.getTitle(sHtml, reglist.RegBookName);
        
        //获得所有的章节列表
        Match match = Regex.Match(sHtml, reglist.RegChapterList, RegexOptions.IgnoreCase | RegexOptions.Multiline);
        int iOrder = 0; //计数
        string sURL="";
        string sTitle="";
        string sReplaceContent="";
        //URL列表
        ArrayList asURL=new ArrayList();
        //标题列表
        ArrayList asTitle = new ArrayList();


        while (match.Success)
        {
            sURL = match.Groups[1].Value;
            sTitle = match.Groups[2].Value.Trim();
            sTitle = Regex.Replace(sTitle, "[\\s]*\\s", " ", RegexOptions.IgnoreCase | RegexOptions.Multiline);

            asURL.Add(sURL);
            asTitle.Add(sTitle);

            sReplaceContent += "<div><a title=\"" + sURL + "\">" + sTitle + "</a></div>\r\n";

            match = match.NextMatch();
            iOrder++;
        }
        
        //插入新章节章节表里面。
        insertNewPage(info,asURL,asTitle);
        //保存起来
        StreamReader sr = new StreamReader(sListModelMapPath);
        string sModel = sr.ReadToEnd();
        sr.Close();

        sModel = sModel.Replace("<#TITLE>", sBookName);
        sModel = sModel.Replace("<#DATA>", sReplaceContent);

        //处理的页面
        string sPageProcess = "/PageClear.aspx?pagename=" + "http://" + sPathName;

        sModel = sModel.Replace("<#PAGE>", sPageProcess);

        if (!Directory.Exists(sDirMapPath))
            Directory.CreateDirectory(sDirMapPath);
        //写入保存文件
        StreamWriter sw = new StreamWriter(sFileMapPath, false, Encoding.UTF8);
        sw.WriteLine(sModel);
        sw.Close();

        return sBookName;
    }
    /// <summary>
    /// 插入新纪录
    /// </summary>
    /// <param name="info"></param>
    /// <param name="asURL"></param>
    /// <param name="asTitle"></param>
    public static void insertNewPage(IndexInfo info,ArrayList asURL,ArrayList asTitle)
    {
        SqlDataSource ds = new SqlDataSource();
        ds.ConnectionString = System.Web.Configuration.WebConfigurationManager.ConnectionStrings["ls_Database"].ToString();
        ds.DeleteCommand = "delete from ls_ChapterT where ChapterT_ListID=@listid";
        ds.DeleteParameters.Add("listid",TypeCode.Int32, info.IndexID);
        ds.Delete();

        ds.InsertCommand = "insert into ls_ChapterT(ChapterT_ListID,ChapterT_URL,ChapterT_Title)values(@listid,@url,@title)";

        for (int i = 0; i < asURL.Count; i++)
        {
            ds.InsertParameters.Clear();
            ds.InsertParameters.Add("listid", TypeCode.Int32, info.IndexID);
            ds.InsertParameters.Add("url", asURL[i].ToString());
            ds.InsertParameters.Add("title", asTitle[i].ToString());
            ds.Insert();
        }
        ds.Dispose();
    }
    /// <summary>
    /// 获得导航
    /// </summary>
    /// <param name="sPageName">当前位置</param>
    /// <param name="sIndexURL">主页</param>
    /// <returns></returns>
    public static GuideURL getGuideURL(string sPageName,IndexInfo info)
    {

        GuideURL guide = new GuideURL();
        SqlDataSource ds = new SqlDataSource();
        ds.ConnectionString = System.Web.Configuration.WebConfigurationManager.ConnectionStrings["ls_Database"].ToString();
        ds.SelectCommand = "select ChapterT_URL from ls_ChapterT where ChapterT_ListID=@listid order by ChapterT_ID";
        ds.SelectParameters.Add("listid",TypeCode.Int32,info.IndexID);

        DataView dv = (DataView)ds.Select(DataSourceSelectArguments.Empty);
        if (dv.Count > 0)
        {
            int iOrder=0;
            for (int i = 0; i < dv.Count; i++)
            {
                if (dv[i]["ChapterT_URL"].ToString() == sPageName)
                {
                    iOrder = i;
                    break;
                }
            }

            //第一页
            guide.FirstPage = dv[0]["ChapterT_URL"].ToString();

            //上一页
            if(iOrder==0)
                guide.LastPage = dv[0]["ChapterT_URL"].ToString();
            else
                guide.LastPage = dv[iOrder-1]["ChapterT_URL"].ToString();

            //下一页
            if (iOrder + 1 == dv.Count) //已经是最后页就返回目录
                guide.NextPage = URLSub.getPageName(info.IndexURL);
            else
                guide.NextPage = dv[iOrder + 1]["ChapterT_URL"].ToString();

            //最后页
            guide.EndPage = dv[dv.Count - 1]["ChapterT_URL"].ToString();

            //目录页
            guide.IndexPage = URLSub.getPageName(info.IndexURL);


        }
        return guide;
    }
    /// <summary>
    /// 获得目录页的记录号
    /// </summary>
    /// <param name="sURL">目录页地址</param>
    /// <returns>记录号</returns>
    public static IndexInfo getIndexInfo(string sURL)
    {
        SqlDataSource ds = new SqlDataSource();
        IndexInfo info=new IndexInfo();
        ds.ConnectionString = System.Web.Configuration.WebConfigurationManager.ConnectionStrings["ls_Database"].ToString();
        ds.SelectCommand = "select * from ls_List where List_URL=@listurl";
        ds.SelectParameters.Add("listurl", sURL);
        DataView dv = (DataView)ds.Select(DataSourceSelectArguments.Empty);
        if (dv.Count == 1)
        {
            info.IndexID = dv[0]["List_ID"].ToString();
            info.IndexURL = sURL;
            info.BookID = dv[0]["List_BookID"].ToString();
            info.BookName = dv[0]["List_BookName"].ToString();
        }
        else
        {
            info.IndexID = "0";
            info.IndexURL = "/index.htm";
            info.BookName = "临时访问,目录导航可能有问题。加入书架后可获得更佳服务。";
            info.BookID = "0";
        }
        ds.Dispose();
        dv.Dispose();
        return info;
    }
   
}
